const cheerio = require("cheerio");
const {get} = require("axios");
const mysql = require('mysql');
const express = require('express')
const app = express()

const links = [],
    NEWS_SITE = 'huanqiu',
    maxClient = 10,
    processor = {
        current: {},
        currentCount: 0,
        success: [],
        failed: []
    };


app.get('/', (req, res) => {
    res.send({
        links,
        processor
    })
})
app.get('/shutdown',(req,res)=>{
    res.send('server will close');
    setTimeout(()=>app.close(),1000);
})
// 创建连接池
const pool = mysql.createPool({
    connectionLimit: maxClient,
    host: 'localhost',
    user: 'root',
    password: '123456',
    database: 'test_news'
});

function foreach(nodes, callback) {
    for (let i = 0; i < nodes.length; i++) {
        const node = cheerio.load(nodes[i]);
        callback.call(node, node, nodes[i]);
    }
}

// eslint-disable-next-line no-unused-vars
function parseCategories($) {
    const categories = $('.newNavBg .newBg .newCon .rTxt a')
    console.log(categories[0].attribs.href)
    for (let i = 0; i < categories.length; i++) {
        const a = cheerio.load(categories[i]);
        const title = a.text();
        let alias = categories[i].attribs.href;//a.attr('href');
        alias = alias.substring(2, alias.indexOf('.'))
        const sql = `insert news_cate ( title, alias) value ('${title}','${alias}');`;
        console.log(sql);
    }
}

// 爬取首页
function getIndex() {
    get('https://www.huanqiu.com/').then(resp => {
        const $ = cheerio.load(resp.data)
        // parseCategories($)
        const allLinks = $('.wrapBg a')
        foreach(allLinks, (link, a) => {
            let href = a.attribs['href'];
            if (href.match(/\/article\/\w+/)) {
                if (!href.startsWith('http')) {
                    href = 'https:' + href
                }
                links.push(href)
            }
        })
        processLinks()

    })// 爬虫
}

function keyExists(key) {
    return new Promise((resolve, reject) => {
        pool.query({
            sql: 'select count(1) as res_count from news where uuid = ?',
            values: [key]
        }, (err, res) => {
            if (err) reject(err)
            else {
                if (res.length > 0) {
                    resolve(res[0]['res_count'] > 0)
                } else resolve(false)

            }
        })
    })
}

function fetchUrl(newsUrl, key) {
    console.log('开始采集 ', newsUrl)
    processor.currentCount++;
    processor.current[newsUrl] = 'processing';
    // 开始获取数据
    get(newsUrl).then(resp => {
        if (resp.status != 200) {
            throw new Error(`采集 ${newsUrl} 失败,状态码为:` + resp.status);
        }
        //
        const $ = cheerio.load(resp.data)
        // 获取标题 内容等信息
        let title = $('.t-container .t-container-title h3').text().trim(),
            source = $('.metadata-info span.source a').text().trim(),
            pubTime = $('.metadata-info p.time').text().trim(),
            author = $('.metadata-info span.author').text().trim(),
            content = $('.b-container .l-container article').html();
        if(author.length > 0 && author.indexOf('：') != -1){
            author = author.replace('作者：','');
        }
        if(pubTime.length == 16){
            pubTime += ':00'
        }
        const cate = newsUrl.substring(newsUrl.indexOf('//') + 2, newsUrl.indexOf('.'));
        saveNews(title, source, pubTime, author, content, cate, key)
        console.log('采集 ' + newsUrl + ' 成功')
    }).catch(err => {
// 采集失败的处理
        console.log(err)
        processor.failed.push(newsUrl)
    }).finally(() => {
        // 处理后续操作
        // 删除当前数据
        delete processor.current[newsUrl]
        //
        processor.currentCount--
        processLinks()
    })
}

// 开始采集
function startCrawl(newsUrl) {
    if(!newsUrl){
        processLinks()
        return;
    }
    const key = NEWS_SITE + ":" + newsUrl.substr(newsUrl.lastIndexOf('/') + 1)

    keyExists(key).then(exists => {
        if (exists) {
            processLinks();
            return;
        }
        fetchUrl(newsUrl, key)
    })

}

function saveNews(title, source, pubTime, author, content, cate, news_key = '') {
    const sql = `insert into news (cate, title, content, source, author, pub_time, uuid) value (?,?,?,?,?,?,?)`;
    pool.query({
        sql,
        values: [cate, title, content, source, author, pubTime, news_key]
    }, (err) => {
        if (err) {
            console.log('保存成功');
        }
    })
}

app.listen(3000);
function processLinks() {
    if (links.length == 0 && processor.currentCount == 0) {
        pool.end();
        console.log('采集完成')
        if(processor.failed.length > 0){
            console.log('------------- 采集失败如下 ----------------')
            processor.failed.forEach(link => console.log(link))
        }
        return;
    }
    if (links.length > 0) {
        for (let i = 0; i < maxClient - processor.currentCount; i++) {
            startCrawl(links.shift())
        }
    }
}

getIndex();